#!/var/bin/python
# -*- coding: utf-8 -*-
#######################
#	获取企查查企业基础信息
#######################

import sys
import time
import urllib,urllib.request
import re
from bs4 import BeautifulSoup
import io
import os
dirs = os.path.abspath(os.path.dirname(__file__)+"/../Config")
os.sys.path.append(dirs)   #将上上级目录加载到python的环境变量中
# os.sys.path.append("D:/job/crawler/Config")
from config import conn,webdriver,logpath,Proxy,ProxyType
from selenium.webdriver.common.keys import Keys  #selenium 输入需要引入keys包

##多进程
from multiprocessing import Pool
import random

import socket
ip = socket.gethostbyname(socket.gethostname())

print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),'Start crawling the data. . .')

cur = conn.cursor()
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"Open your browser. . .")
iplist = []
getrandomiptime = 0

def getrandomip():
	global getrandomiptime
	global iplist
	if int(time.time()) - getrandomiptime>900:
		data = urllib.request.urlopen("http://api.xicidaili.com/free2016.txt").read()
		z_data = data.decode('UTF-8')
		getrandomiptime = int(time.time())
		iplist = z_data.split('\r\n')
	return iplist

def getQichacha(name,driver):
	time.sleep(5)
	# phantomjspath = "D:/phantomjs%s/bin/phantomjs" % name
	# driver = webdriver.Firefox()
	# # 利用DesiredCapabilities(代理设置)参数值，重新打开一个sessionId，我看意思就相当于浏览器清空缓存后，加上代理重新访问一次url
	# proxy = webdriver.Proxy()
	# proxy.proxy_type = ProxyType.MANUAL
	# # proxy.http_proxy = random.choice(getrandomip())
	# proxy.http_proxy = "101.254.188.198:8080"
	# # 将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中
	# proxy.add_to_capabilities(webdriver.DesiredCapabilities.FIREFOX)
	# driver.start_session(webdriver.DesiredCapabilities.FIREFOX)
	driver.implicitly_wait(120)

	# # 打开登陆页面
	# print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Open the login page. . .", name)
	# driver.get(
	# 	"https://graph.qq.com/oauth/show?which=Login&display=pc&response_type=code&client_id=101188807&redirect_uri=http%3A%2F%2Fwww.qichacha.com%2Fuser_callbackqq&state=3daff0f5935d33e95facd2033cdbb312&scope=get_user_info,add_share")
	# driver.switch_to_frame('ptlogin_iframe')
    #
	# print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Switch to the use of account password login. . .",
	# 	  name)
	# driver.find_element_by_id('switcher_plogin').click()
	# print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Enter QQ account password. . .", name)
	# driver.find_element_by_id("u").clear()
	# driver.find_element_by_id("u").send_keys("2127483086")
	# driver.find_element_by_id("p").clear()
	# driver.find_element_by_id("p").send_keys("123698745")
	# driver.find_element_by_id("login_button").click()
	# time.sleep(3)

	#获取本进程查询的范围
	searchname = ""
	sqlname = "select ename from enterpriselist where status=1 order by id asc limit 1"
	cur.execute(sqlname)
	res = cur.fetchone()
	if res != None:
		searchname = res[0]
	else:
		print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"All Completed. . .")
		exit()

	#加个状态锁
	sqlname = "update enterpriselist set status=2 where ename='%s'" % searchname
	cur.execute(sqlname)
	conn.commit()
	#加个状态锁
	handles = driver.window_handles # 获取当前窗口句柄集合（列表类型）
	driver.switch_to_window(handles[-1])
	time.sleep(5)
	print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"Search in the home page. . .",name)
	try:
		driver.get("http://www.qichacha.com")
		driver.find_element_by_name("key").clear()
		driver.find_element_by_name("key").send_keys(searchname)
		driver.find_element_by_name("key").send_keys(Keys.ENTER)
		# driver.get("http://www.qichacha.com/search?key=%s" % searchname)
		time.sleep(3)
		try:
			print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"Open the Company page. . .",name)
			driver.find_element_by_id("searchlist").find_element_by_tag_name("a").click()
			time.sleep(3)
			handles = driver.window_handles # 获取当前窗口句柄集合（列表类型）
			driver.switch_to_window(handles[-1])
			soup = BeautifulSoup(driver.page_source,"html.parser")
			base_lis = soup.find("section",id="Cominfo").find_all("td",class_="ma_left")

			base = {}
			baseinfolist = []
			columnname = []
			columnval = []
			for base_li in base_lis:
				if base_lis.index(base_li) % 2 == 0:
					columnname.append(base_li)
				else:
					columnval.append(base_li)
			for j in range(len(columnname)):
				baseinfolist.append(columnname[j].text.strip()+columnval[j].text.strip())
			for columninfo in baseinfolist:
				base_liarr = columninfo.split("：")
				base[base_liarr[0].strip()] = base_liarr[1].strip()

			fullname = soup.find("span",class_="text-big").text.strip()
			#判断是否已经存在数据
			sqls = "select * from enterpriseinfo where ename like '%s'" % ("%"+fullname+"%")
			cur.execute(sqls)
			result = cur.fetchone()
			print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"data processing. . .",name)
			if result:
				sql = "update enterpriseinfo set `searchname`='%s',`ename`='%s',`creditcode`='%s',`regnumber`='%s',`organizationcode`='%s',`managementforms`='%s',`companytype`='%s',`RegisterDate`='%s',`legalperson`='%s',`registeredcapital`='%s',`busnissallotedtime`='%s',`registeroffice`='%s',`dateofissue`='%s',`scale`='%s',`industry`='%s',`Englishname`='%s',`businessaddress`='%s',`Businessscope`='%s',`status`=1,`ip`='%s',`QQ`='%s' where eid=%d" % (searchname,fullname,base.get("统一社会信用代码",""),base.get("注册号",""),base.get("组织机构代码",""),base.get("经营状态",""),base.get("公司类型",""),base.get("成立日期",""),base.get("法定代表人",""),base.get("注册资本",""),base.get("营业期限",""),base.get("登记机关",""),base.get("发照日期",""),base.get("公司规模",""),base.get("所属行业",""),base.get("英文名",""),base.get("企业地址",""),base.get("经营范围",""),ip,QQ,result[0])
			else:
				sql = "insert into enterpriseinfo (`searchname`,`ename`,`creditcode`,`regnumber`,`organizationcode`,`managementforms`,`companytype`,`RegisterDate`,`legalperson`,`registeredcapital`,`busnissallotedtime`,`registeroffice`,`dateofissue`,`scale`,`industry`,`Englishname`,`businessaddress`,`Businessscope`,`status`,`ip`,`QQ`) values ('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s',%d,'%s','%s')" % (searchname,fullname,base.get("统一社会信用代码",""),base.get("注册号",""),base.get("组织机构代码",""),base.get("经营状态",""),base.get("公司类型",""),base.get("成立日期",""),base.get("法定代表人",""),base.get("注册资本",""),base.get("营业期限",""),base.get("登记机关",""),base.get("发照日期",""),base.get("公司规模",""),base.get("所属行业",""),base.get("英文名",""),base.get("企业地址",""),base.get("经营范围",""),1,ip,QQ)
			
			try:
				cur.execute(sql)
				conn.commit()
				print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"Add company basic data to the database",name)
				#解个状态锁 到完成状态
				sqlname = "update enterpriselist set status=3 where ename='%s'" % searchname
				cur.execute(sqlname)
				conn.commit()
				#解个状态锁 到完成状态
				driver.close()
			except Exception as e:
				print("Failed to add data...",sql,e,name)

				#解个状态锁 到初始状态
				sqlname = "update enterpriselist set status=3 where ename='%s'" % searchname
				cur.execute(sqlname)
				conn.commit()
				#解个状态锁 到初始状态
				driver.close()
		except Exception as e:
			print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"data not found...",e,name)
			#解个状态锁 到未找到的状态
			sqlname = "update enterpriselist set status=3 where ename='%s'" % searchname
			cur.execute(sqlname)
			conn.commit()
			#解个状态锁 到未找到的状态
	except Exception as e:
		print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Unknown Error", e, name)
		# 解个状态锁 到初始状态
		sqlname = "update enterpriselist set status=1 where ename='%s'" % searchname
		cur.execute(sqlname)
		conn.commit()
		# 解个状态锁 到初始状态
		driver.close()
		driver.quit()
		return dealData(name)
	# handles = driver.window_handles # 获取当前窗口句柄集合（列表类型）
	# # if len(handles)>2:
	# # 	for windowsid in handles[1:]:
	# # 		print(windowsid)
	# # 		driver.close(windowsid)
	# driver.switch_to_window(handles[-1])
	return getQichacha(name,driver)

#获取详请页面内容 处理
def dealData(name):
	phantomjspath = "D:/phantomjs%s/bin/phantomjs" % name
	# driver = webdriver.Firefox()
	driver = webdriver.PhantomJS(executable_path=phantomjspath)
	# # 利用DesiredCapabilities(代理设置)参数值，重新打开一个sessionId，我看意思就相当于浏览器清空缓存后，加上代理重新访问一次url
	# proxy = webdriver.Proxy()
	# proxy.proxy_type = ProxyType.MANUAL
	# # proxy.http_proxy = random.choice(getrandomip())
	# proxy.http_proxy = "183.141.154.0:3128"
	# # 将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中
	# proxy.add_to_capabilities(webdriver.DesiredCapabilities.FIREFOX)
	# driver.start_session(webdriver.DesiredCapabilities.FIREFOX)
	driver.implicitly_wait(120)

	# 打开登陆页面
	print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Open the login page. . .", name)
	driver.get("https://graph.qq.com/oauth/show?which=Login&display=pc&response_type=code&client_id=101188807&redirect_uri=http%3A%2F%2Fwww.qichacha.com%2Fuser_callbackqq&state=3daff0f5935d33e95facd2033cdbb312&scope=get_user_info,add_share")
	driver.switch_to_frame('ptlogin_iframe')

	print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Switch to the use of account password login. . .",
		  name)
	driver.find_element_by_id('switcher_plogin').click()
	print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), "Enter QQ account password. . .", name)
	driver.find_element_by_id("u").clear()
	qqlist = {
		"3526641052":"123698745",
		"2127483086":"123698745",
		"":"",
		"":"",
		"":""
	}
	driver.find_element_by_id("u").send_keys("2252875088")
	driver.find_element_by_id("p").clear()
	driver.find_element_by_id("p").send_keys("123698745")
	driver.find_element_by_id("login_button").click()
	time.sleep(3)
	the_page = BeautifulSoup(driver.page_source,"html.parser")
	pos = str(the_page).find("15042526")
	if pos > -1:
		getQichacha(name,driver)
	else:
		driver.close()
		driver.quit()
		return dealData(name)
	
QQ = "2252875088"
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"Start crawling company basic data. . .")
dealData(0)
exit()
def long_time_task(name,):	
	print('Run task %s (%s)...' % (name, os.getpid()))
	start = time.time()
	time.sleep(random.random() * 3)
	end = time.time()
	print('Task %s runs %0.2f seconds.' % (name, (end - start)))
	dealData(name)
	driver.close()
	driver.quit()

if __name__=='__main__':
	print('Parent process %s.' % os.getpid())
	p = Pool(4)
	for i in range(4):
	    p.apply_async(long_time_task, args=(i,))
	print('Waiting for all subprocesses done...')
	p.close()
	p.join()
	print('All subprocesses done.')
#多进程

